<!doctype html><html lang=zh-hans>
<head>
<meta charset=utf-8>
<meta name=viewport content="width=device-width,initial-scale=1">
<meta http-equiv=x-ua-compatible content="IE=edge">
<meta name=author content="小李刀刀">
<meta name=description content="Spiral 队列和 RoadRunner 服务器可以用来实现不同于传统 web 设置的应用程序。在这篇教程中，我们来演示开发一个在命令行下使用的简单网站爬虫程序。 爬取到的数据会存储在 runtime 文件夹中。 教程开发的代码只是为了进行基础演示，有很多需要改进的地方，不建议直接使用。 安装依赖 我们会基于 Spiral 的命令行项目框架 spiral/app-cli 创建这个命令行">
<meta name=theme-color content="#3f51b5">
<link rel=stylesheet href=https://cdnjs.cloudflare.com/ajax/libs/academicons/1.8.6/css/academicons.min.css integrity="sha256-uFVgMKfistnJAfoCUQigIl+JfUaP47GrRKjf6CTPVmw=" crossorigin=anonymous>
<link rel=stylesheet href=https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.0-1/css/all.min.css integrity="sha256-4w9DunooKSr3MFXHXWyFER38WmPdm361bQS/2KUWZbU=" crossorigin=anonymous>
<link rel=stylesheet href=https://cdnjs.cloudflare.com/ajax/libs/fancybox/3.5.7/jquery.fancybox.min.css integrity="sha256-Vzbj7sDDS/woiFS3uNKo8eIuni59rjyNGtXfstRzStA=" crossorigin=anonymous>
<link rel=stylesheet href=https://cdnjs.cloudflare.com/ajax/libs/highlight.js/9.18.1/styles/tomorrow-night.min.css crossorigin=anonymous title=hl-light>
<link rel=stylesheet href=https://cdnjs.cloudflare.com/ajax/libs/highlight.js/9.18.1/styles/tomorrow-night.min.css crossorigin=anonymous title=hl-dark disabled>
<script src=https://cdnjs.cloudflare.com/ajax/libs/lazysizes/5.1.2/lazysizes.min.js integrity="sha256-Md1qLToewPeKjfAHU1zyPwOutccPAm5tahnaw7Osw0A=" crossorigin=anonymous async></script>
<link rel=stylesheet href="https://fonts.font.im/css?family=Source+Code+Pro:300,400,500,600,700&subset=latin-ext&display=swap">
<link rel=stylesheet href=/css/academic.css>
<script>(function(b,d,e,a,g){b[a]=b[a]||[],b[a].push({'gtm.start':(new Date).getTime(),event:'gtm.js'});var f=d.getElementsByTagName(e)[0],c=d.createElement(e),h=a!='dataLayer'?'&l='+a:'';c.async=!0,c.src='https://www.googletagmanager.com/gtm.js?id='+g+h,f.parentNode.insertBefore(c,f)})(window,document,'script','dataLayer','GTM-5KZH8N7')</script>
<link rel=manifest href=/index.webmanifest>
<link rel=icon type=image/png href=/images/icon_hu0b7a4cb9992c9ac0e91bd28ffd38dd00_9727_32x32_fill_lanczos_center_3.png>
<link rel=apple-touch-icon type=image/png href=/images/icon_hu0b7a4cb9992c9ac0e91bd28ffd38dd00_9727_192x192_fill_lanczos_center_3.png>
<link rel=canonical href=https://studyspiral.cn/docs/queue/scraper/>
<meta property="twitter:card" content="summary">
<meta property="og:site_name" content="Spiral中文网">
<meta property="og:url" content="https://studyspiral.cn/docs/queue/scraper/">
<meta property="og:title" content="网站爬虫 | Spiral中文网">
<meta property="og:description" content="Spiral 队列和 RoadRunner 服务器可以用来实现不同于传统 web 设置的应用程序。在这篇教程中，我们来演示开发一个在命令行下使用的简单网站爬虫程序。 爬取到的数据会存储在 runtime 文件夹中。 教程开发的代码只是为了进行基础演示，有很多需要改进的地方，不建议直接使用。 安装依赖 我们会基于 Spiral 的命令行项目框架 spiral/app-cli 创建这个命令行"><meta property="og:image" content="https://studyspiral.cn/images/logo.svg">
<meta property="twitter:image" content="https://studyspiral.cn/images/logo.svg"><meta property="og:locale" content="zh-Hans">
<meta property="article:published_time" content="2020-03-01T22:58:55+08:00">
<meta property="article:modified_time" content="2020-03-25T03:56:47+08:00">
<script src=https://cdnjs.cloudflare.com/ajax/libs/cookieconsent2/3.1.1/cookieconsent.min.js integrity="sha256-5VhCqFam2Cn+yjw61zbBNrbHVJ6SRydPeKopYlngbiQ=" crossorigin=anonymous></script>
<link rel=stylesheet href=https://cdnjs.cloudflare.com/ajax/libs/cookieconsent2/3.1.1/cookieconsent.min.css integrity="sha256-zQ0LblD/Af8vOppw18+2anxsuaz3pWYyVWi+bTvTH8Q=" crossorigin=anonymous>
<script>window.addEventListener("load",function(){window.cookieconsent.initialise({palette:{popup:{background:"#3f51b5",text:"#fff"},button:{background:"#fff",text:"#3f51b5"}},theme:"classic",content:{message:"本网站使用cookies来确保您在本网站上获得最佳体验。",dismiss:"知道了!",link:"了解更多",href:"https://www.cookiesandyou.com"}})})</script>
<title>网站爬虫 | Spiral中文网</title>
</head>
<body id=top data-spy=scroll data-offset=70 data-target=#TableOfContents>
<aside class=search-results id=search>
<div class=container>
<section class=search-header>
<div class="row no-gutters justify-content-between mb-3">
<div class=col-6>
<h1>搜索</h1>
</div>
<div class="col-6 col-search-close">
<a class=js-search href=#><i class="fas fa-times-circle text-muted" aria-hidden=true></i></a>
</div>
</div>
<div id=search-box>
<input name=q id=search-query placeholder=搜索... autocapitalize=off autocomplete=off autocorrect=off spellcheck=false type=search>
</div>
</section>
<section class=section-search-results>
<div id=search-hits>
</div>
</section>
</div>
</aside>
<nav class="navbar navbar-expand-lg navbar-light compensate-for-scrollbar" id=navbar-main>
<div class=container>
<div class="d-none d-lg-inline-flex">
<a class=navbar-brand href=/><img src=/images/logo.svg alt=Spiral中文网>Spiral中文网</a>
</div>
<button type=button class=navbar-toggler data-toggle=collapse data-target=#navbar-content aria-controls=navbar aria-expanded=false aria-label=切换导航>
<span><i class="fas fa-bars"></i></span>
</button>
<div class="navbar-brand-mobile-wrapper d-inline-flex d-lg-none">
<a class=navbar-brand href=/><img src=/images/logo.svg alt=Spiral中文网>Spiral中文网</a>
</div>
<div class="navbar-collapse main-menu-item collapse justify-content-end" id=navbar-content>
<ul class="navbar-nav d-md-inline-flex">
<li class=nav-item>
<a class=nav-link href=/docs/basics/quick-start/><span>教程</span></a>
</li>
<li class=nav-item>
<a class="nav-link active" href=/docs/><span>文档</span></a>
</li>
<li class=nav-item>
<a class=nav-link href=/post/><span>文章</span></a>
</li>
</ul>
</div>
<ul class="nav-icons navbar-nav flex-row ml-auto d-flex pl-md-2">
<li class=nav-item>
<a class="nav-link js-search" href=#><i class="fas fa-search" aria-hidden=true></i></a>
</li>
</ul>
</div>
</nav>
<div class="container-fluid docs">
<div class="row flex-xl-nowrap">
<div class="col-12 col-md-3 col-xl-2 docs-sidebar">
<form class="docs-search d-flex align-items-center">
<button class="btn docs-toggle d-md-none p-0 mr-3" type=button data-toggle=collapse data-target=#docs-nav aria-controls=docs-nav aria-expanded=false aria-label="Toggle section navigation">
<span><i class="fas fa-bars"></i></span>
</button>
<input name=q type=search class=form-control placeholder=搜索... autocomplete=off>
</form>
<nav class="collapse docs-links" id=docs-nav>
<div class=docs-toc-item>
<a class=docs-toc-link href=/docs/extension/dotenv/>Dotenv</a>
</div>
<div class=docs-toc-item>
<a class=docs-toc-link href=/docs/>总览</a>
<ul class="nav docs-sidenav docs-sidenav-sub collapse show">
<li>
<a href=/docs/>Spiral介绍</a>
</li>
<li>
<a href=/docs/about/contributing/>贡献指引</a>
</li>
<li>
<a href=/docs/about/semver/>版本说明</a>
</li>
<li>
<a href=/docs/about/license/>授权协议</a>
</li>
</ul>
</div>
<div class=docs-toc-item>
<a class=docs-toc-link href=/docs/http/psr-15/>定制 PSR-15 处理器</a>
</div>
<div class=docs-toc-item>
<a class=docs-toc-link href=/docs/start/install/>快速开始</a>
<ul class="nav docs-sidenav docs-sidenav-sub collapse">
<li>
<a href=/docs/start/install/>安装指引</a>
</li>
<li>
<a href=/docs/start/workers/>应用工作进程</a>
</li>
<li>
<a href=/docs/start/structure/>应用程序结构</a>
</li>
<li>
<a href=/docs/start/configuration/>默认配置</a>
</li>
<li>
<a href=/docs/start/commands/>控制台命令</a>
</li>
</ul>
</div>
<div class=docs-toc-item>
<a class=docs-toc-link href=/docs/basics/quick-start/>入门基础</a>
<ul class="nav docs-sidenav docs-sidenav-sub collapse">
<li>
<a href=/docs/basics/quick-start/>上手指南</a>
</li>
<li>
<a href=/docs/basics/scaffolding/>脚手架</a>
</li>
<li>
<a href=/docs/basics/prototype/>原型开发辅助</a>
</li>
</ul>
</div>
<div class=docs-toc-item>
<a class=docs-toc-link href=/docs/framework/design/>核心框架</a>
<ul class="nav docs-sidenav docs-sidenav-sub collapse">
<li>
<a href=/docs/framework/design/>设计理念</a>
</li>
<li>
<a href=/docs/framework/application-server/>应用服务器</a>
</li>
<li>
<a href=/docs/framework/config/>配置对象</a>
</li>
<li>
<a href=/docs/framework/kernel/>内核与环境</a>
</li>
<li>
<a href=/docs/framework/container/>容器</a>
</li>
<li>
<a href=/docs/framework/bootloaders/>引导程序</a>
</li>
<li>
<a href=/docs/framework/scopes/>IoC 作用域</a>
</li>
<li>
<a href=/docs/framework/memory/>静态高速缓存</a>
</li>
<li>
<a href=/docs/framework/finalizers/>终结器</a>
</li>
</ul>
</div>
<div class=docs-toc-item>
<a class=docs-toc-link href=/docs/cookbook/annotated-routes/>速查手册</a>
<ul class="nav docs-sidenav docs-sidenav-sub collapse">
<li>
<a href=/docs/cookbook/annotated-routes/>注解式路由</a>
</li>
<li>
<a href=/docs/cookbook/injector/>容器注入器</a>
</li>
<li>
<a href=/docs/cookbook/domain-core/>领域内核、控制器</a>
</li>
<li>
<a href=/docs/cookbook/golang-library/>Golang服务集成</a>
</li>
<li>
<a href=/docs/cookbook/custom-dispatcher/>自定义调度器</a>
</li>
</ul>
</div>
<div class=docs-toc-item>
<a class=docs-toc-link href=/docs/component/files/>常用组件</a>
<ul class="nav docs-sidenav docs-sidenav-sub collapse">
<li>
<a href=/docs/component/files/>文件和目录</a>
</li>
<li>
<a href=/docs/component/reactor/>代码生成</a>
</li>
<li>
<a href=/docs/component/tokenizer/>静态分析工具</a>
</li>
<li>
<a href=/docs/component/metrics/>应用指标</a>
</li>
<li>
<a href=/docs/component/data-grid/>数据网格</a>
</li>
</ul>
</div>
<div class=docs-toc-item>
<a class=docs-toc-link href=/docs/console/configuration/>控制台</a>
<ul class="nav docs-sidenav docs-sidenav-sub collapse">
<li>
<a href=/docs/console/configuration/>安装和配置</a>
</li>
<li>
<a href=/docs/console/commands/>用户命令</a>
</li>
</ul>
</div>
<div class=docs-toc-item>
<a class=docs-toc-link href=/docs/http/configuration/>HTTP</a>
<ul class="nav docs-sidenav docs-sidenav-sub collapse">
<li>
<a href=/docs/http/configuration/>安装配置</a>
</li>
<li>
<a href=/docs/http/lifecycle/>请求生命周期</a>
</li>
<li>
<a href=/docs/http/request-response/>请求和响应</a>
</li>
<li>
<a href=/docs/http/routing/>路由</a>
</li>
<li>
<a href=/docs/http/errors/>错误页面</a>
</li>
<li>
<a href=/docs/http/middleware/>中间件</a>
</li>
<li>
<a href=/docs/http/golang/>Golang 中间件</a>
</li>
<li>
<a href=/docs/http/cookies/>Cookies</a>
</li>
<li>
<a href=/docs/http/session/>Session</a>
</li>
<li>
<a href=/docs/http/csrf/>CSRF 防护</a>
</li>
</ul>
</div>
<div class=docs-toc-item>
<a class=docs-toc-link href=/docs/security/encrypter/>安全</a>
<ul class="nav docs-sidenav docs-sidenav-sub collapse">
<li>
<a href=/docs/security/encrypter/>数据加密</a>
</li>
<li>
<a href=/docs/security/validation/>数据验证</a>
</li>
<li>
<a href=/docs/security/rbac/>基于角色的权限控制</a>
</li>
<li>
<a href=/docs/security/authentication/>用户认证</a>
</li>
</ul>
</div>
<div class=docs-toc-item>
<a class=docs-toc-link href=/docs/filters/configuration/>请求过滤</a>
<ul class="nav docs-sidenav docs-sidenav-sub collapse">
<li>
<a href=/docs/filters/configuration/>安装和配置</a>
</li>
<li>
<a href=/docs/filters/filter/>过滤器</a>
</li>
<li>
<a href=/docs/filters/composite/>复合过滤器</a>
</li>
</ul>
</div>
<div class=docs-toc-item>
<a class=docs-toc-link href=/docs/database/configuration/>数据库</a>
<ul class="nav docs-sidenav docs-sidenav-sub collapse">
<li>
<a href=/docs/database/configuration/>安装与配置</a>
</li>
<li>
<a href=/docs/database/access/>访问数据</a>
</li>
<li>
<a href=/docs/database/isolation/>逻辑隔离</a>
</li>
<li>
<a href=/docs/database/query-builders/>查询构造器</a>
</li>
<li>
<a href=/docs/database/transactions/>Transactions</a>
</li>
<li>
<a href=/docs/database/introspection/>Schema Introspection</a>
</li>
<li>
<a href=/docs/database/declaration/>Declaration</a>
</li>
<li>
<a href=/docs/database/migrations/>Migrations</a>
</li>
<li>
<a href=/docs/database/errata/>Errata</a>
</li>
</ul>
</div>
<div class=docs-toc-item>
<a class=docs-toc-link href=/docs/cycle/configuration/>Cycle ORM</a>
<ul class="nav docs-sidenav docs-sidenav-sub collapse">
<li>
<a href=/docs/cycle/configuration/>Configuration</a>
</li>
</ul>
</div>
<div class=docs-toc-item>
<a class=docs-toc-link href=/docs/queue/configuration/>队列任务</a>
<ul class="nav docs-sidenav docs-sidenav-sub collapse show">
<li>
<a href=/docs/queue/configuration/>Configuration</a>
</li>
<li class=active>
<a href=/docs/queue/scraper/>网站爬虫</a>
</li>
</ul>
</div>
<div class=docs-toc-item>
<a class=docs-toc-link href=/docs/views/configuration/>视图</a>
<ul class="nav docs-sidenav docs-sidenav-sub collapse">
<li>
<a href=/docs/views/configuration/>Configuration</a>
</li>
</ul>
</div>
<div class=docs-toc-item>
<a class=docs-toc-link href=/docs/stempler/configuration/>Stempler 模板</a>
<ul class="nav docs-sidenav docs-sidenav-sub collapse">
<li>
<a href=/docs/stempler/configuration/>Configuration</a>
</li>
<li>
<a href=/docs/stempler/directives/>Directives</a>
</li>
</ul>
</div>
<div class=docs-toc-item>
<a class=docs-toc-link>国际化</a>
</div>
<div class=docs-toc-item>
<a class=docs-toc-link>GRPC</a>
</div>
<div class=docs-toc-item>
<a class=docs-toc-link>事件广播</a>
</div>
<div class=docs-toc-item>
<a class=docs-toc-link>调试及性能</a>
</div>
<div class=docs-toc-item>
<a class=docs-toc-link>扩展</a>
</div>
</nav>
</div>
<div class="d-none d-xl-block col-xl-2 docs-toc">
<ul class="nav toc-top">
<li><a href=# id=back_to_top class=docs-toc-title>在本页</a></li>
</ul>
<nav id=TableOfContents>
<ul>
<li><a href=#安装依赖>安装依赖</a></li>
<li><a href=#配置应用服务器>配置应用服务器</a></li>
<li><a href=#创建任务处理器>创建任务处理器</a></li>
<li><a href=#创建命令>创建命令</a></li>
<li><a href=#测试一下>测试一下</a></li>
</ul>
</nav>
</div>
<main class="col-12 col-md-9 col-xl-8 py-md-3 pl-md-5 docs-content" role=main>
<article class=article>
<div class="alert alert-warning" role=alert>
官方文档中文版翻译工作仍在进行中，欢迎 <a href=/post/join-translation/>参与翻译</a>。
</div>
<div class=docs-article-container>
<h1>网站爬虫</h1>
<div class=article-style>
<p>Spiral 队列和 RoadRunner 服务器可以用来实现不同于传统 web 设置的应用程序。在这篇教程中，我们来演示开发一个在命令行下使用的简单网站爬虫程序。</p>
<p>爬取到的数据会存储在 <code>runtime</code> 文件夹中。</p>
<blockquote>
<p>教程开发的代码只是为了进行基础演示，有很多需要改进的地方，不建议直接使用。</p>
</blockquote>
<h2 id=安装依赖>安装依赖</h2>
<p>我们会基于 Spiral 的命令行项目框架
<a href=https://github.com/spiral/app-cli target=_blank rel=noopener>spiral/app-cli</a> 创建这个命令行程序。<code>app-cli</code> 是 Spiral 提供的不含 ORM, HTTP，以及其它扩展的精简项目模板，通常用于开发控制台应用程序。</p>
<p>执行下面的命令创建新项目：</p>
<pre><code class=language-bash>$ composer create-project spiral/app-cli scraper
$ cd scraper
</code></pre>
<p>为了实现网站爬虫所需的功能，要用到的扩展组件包括这些：</p>
<table>
<thead>
<tr>
<th>组件</th>
<th>说明</th>
</tr>
</thead>
<tbody>
<tr>
<td>spiral/jobs</td>
<td>队列任务支持</td>
</tr>
<tr>
<td>spiral/scaffolder</td>
<td>快速脚手架（开发环境用）</td>
</tr>
<tr>
<td>spiral/prototype</td>
<td>快速原型开发辅助（开发环境用）</td>
</tr>
<tr>
<td>paquettg/php-html-parser</td>
<td>解析 HTML</td>
</tr>
</tbody>
</table>
<p>首先安装依赖项，并下载应用服务器：</p>
<pre><code class=language-bash>$ composer require spiral/jobs paquettg/php-html-parser spiral/scaffolder spiral/prototype
$ ./vendor/bin/spiral get 
</code></pre>
<p>在 <code>App\App</code> 类中激活安装的扩展组件：</p>
<pre><code class=language-php>namespace App;

use Spiral\Bootloader;
use Spiral\Framework\Kernel;
use Spiral\Prototype\Bootloader as Prototype;
use Spiral\Scaffolder\Bootloader\ScaffolderBootloader;

class App extends Kernel
{
    protected const LOAD = [
        Bootloader\DebugBootloader::class,
        Bootloader\CommandBootloader::class,
        Bootloader\Jobs\JobsBootloader::class,

        // 以下两个依赖在开发完成后可以移除
        ScaffolderBootloader::class,
        Prototype\PrototypeBootloader::class
    ];
}
</code></pre>
<p>配置完成后记得要执行一次 <code>php app.php configure</code> 来确保项目已经正确安装。</p>
<pre><code class=language-bash>$ php app.php conf
Configuring project:

[runtime] verify `runtime` directory access
Verifying runtime directory... exists
Runtime directory permissions were updated.

All done!
</code></pre>
<h2 id=配置应用服务器>配置应用服务器</h2>
<p>作为演示，我们使用默认的一个内存队列。在项目的根目录下创建 <code>.rr.yaml</code> 文件：</p>
<pre><code class=language-yaml>jobs:
  dispatch:
    # 所有 App\Job\* 任务都分发到 local 队列
    app-job-*.pipeline: &quot;local&quot;

  pipelines:
    # 把 local 队列关联到 `ephemeral` 队列服务器（内置的基于内存的队列服务器）
    local.broker: &quot;ephemeral&quot;
  
  # 消费 local 队列
  consume: [&quot;local&quot;]
  
  # 进程命令和进程数
  workers:
    command: &quot;php app.php&quot;
  
    # 对 IO 密集型应用程序，可以适当增加工作进程数量
    pool.numWorkers: 16
</code></pre>
<h2 id=创建任务处理器>创建任务处理器</h2>
<p>接下来创建一个简单的任务处理器，它负责扫描网站，爬取 HTML 内容，并在指定的深度限制范围内跟随网页中的链接跳转和爬取。所有抓到的内容会存储在 <code>runtime</code> 文件夹下。</p>
<blockquote>
<p>因为安装了命令脚手架，可以通过 <code>php app.php create:job scrape</code> 来创建任务处理类。为了缩短篇幅，我们的示例中不使用 CURL 而是是用 <code>file_get_contents</code>.</p>
</blockquote>
<pre><code class=language-php>namespace App\Job;

use PHPHtmlParser\Dom;
use Spiral\Jobs\JobHandler;
use Spiral\Prototype\Traits\PrototypeTrait;

class ScrapeJob extends JobHandler
{
    use PrototypeTrait;

    public function invoke(int $depth, string $url): void
    {
        if ($this-&gt;files-&gt;exists(directory('runtime') . md5($url) . '.html')) {
            // 跳过已抓取的页面
            return;
        }

        $body = file_get_contents($url);
        $this-&gt;store($url, $body);

        $dom = new Dom();
        $dom-&gt;load($body);

        foreach ($dom-&gt;find('a') as $a) {
            $next = $this-&gt;nextURL($url, $a-&gt;href);

            if ($next !== null &amp;&amp; $depth &gt; 1) {
                $this-&gt;queue-&gt;push(self::class, ['depth' =&gt; $depth - 1, 'url' =&gt; $next]);
            }
        }
    }

    private function store(string $url, string $body)
    {
        $this-&gt;files-&gt;write(directory('runtime') . md5($url) . '.html', $body);

        $this-&gt;files-&gt;append(
            directory('runtime') . 'scrape.log',
            sprintf(&quot;%s,%s,%s\n&quot;, date('c'), md5($url), $url)
        );
    }

    private function nextURL(string $base, ?string $target): ?string
    {
        if ($target == null) {
            return null;
        }

        $base_url = parse_url($base);
        $target_url = parse_url($target);

        if (isset($target_url['scheme']) &amp;&amp; isset($target_url['host'])) {
            if ($target_url['host'] !== $base_url['host']) {
                // 只抓取与起点同域名下的链接
                return null;
            }

            // url 包含 scheme 和 host 的完整链接直接返回
            return $target;
        }

        if (!isset($target_url['path'])) {
            return null;
        }

        // 这里是把站内相对链接变成完整链接返回
        return sprintf(&quot;%s://%s%s&quot;, $base_url['scheme'], $base_url['host'], $target_url['path']);
    }
}
</code></pre>
<h2 id=创建命令>创建命令</h2>
<p>通过脚手架，执行 <code>php app.php create:command scrape</code> 来创建控制台命令类：</p>
<pre><code class=language-php>namespace App\Command;

use App\Job\ScrapeJob;
use Spiral\Console\Command;
use Spiral\Jobs\QueueInterface;
use Symfony\Component\Console\Input\InputArgument;

class ScrapeCommand extends Command
{
    public const NAME        = 'scrape';
    public const DESCRIPTION = '爬取网页';
    public const ARGUMENTS   = [
        ['url', InputArgument::REQUIRED, '要爬取的 URL'],
        ['depth', InputArgument::OPTIONAL, '要扫描的深度', 10],
    ];

    protected function perform(QueueInterface $queue): void
    {
        $queue-&gt;push(ScrapeJob::class, [
            'url'   =&gt; $this-&gt;argument('url'),
            'depth' =&gt; (int)$this-&gt;argument('depth')
        ]);

        $this-&gt;writeln(&quot;开始爬取!&quot;);
    }
}
</code></pre>
<h2 id=测试一下>测试一下</h2>
<p>首先启动应用服务器：</p>
<pre><code class=language-bash>$ ./spiral serve -v -d
</code></pre>
<blockquote>
<p>这里添加 <code>-v</code> 参数可以输出详细的信息</p>
</blockquote>
<p>另开一个终端，通过命令行命令开始爬取任意的 URL（要保持应用服务器运行中）：</p>
<pre><code class=language-bash>$ php app.php scrape https://some-website.com/ 5
</code></pre>
<blockquote>
<p>上面的命令从 <code>https://some-website.com/</code> 开始，进行深度为 5 的爬取</p>
</blockquote>
<p>应用服务器还提供了以下查看队列状态的命令，可以通过交互式的控制台查看抓取了多少页面：</p>
<pre><code class=language-bash>$ ./spiral jobs:stat -i
</code></pre>
<blockquote>
<p>实例应用会反复爬取同一个页面很多次（由于网页中反复出现相同的链接），实际项目中要使用适当的数据库或者锁定机制来避免这种情况。</p>
</blockquote>
</div>
<div class=article-widget>
<div class=post-nav>
<div class=post-nav-item>
<div class=meta-nav>上一页</div>
<a href=/docs/cookbook/golang-library/ rel=next>速查手册 - Golang 服务集成</a>
</div>
<div class=post-nav-item>
<div class=meta-nav>下一页</div>
<a href=/docs/cookbook/domain-core/ rel=prev>速查手册 - 领域内核</a>
</div>
</div>
</div>
</div>
<div class=body-footer>
<p>最近更新于 2020-03-25</p>
<p class=edit-page>
<a href=https://github.com/krwu/spiraldocs/edit/feat/chinese/zh_CN/queue/scraper.md>
<i class="fas fa-pen pr-2"></i>编辑本页
</a>
</p>
</div>
</article>
<footer class=site-footer>
<p class=powered-by><span class=copyright>© 2019 - 2021 <a href=https://studyspiral.cn/>StudySpiral</a> All rights reserved.</span>
</p>
<p class=powered-by>
<a href=http://www.beian.miit.gov.cn/ target=_blank rel="noopener noreferer">粤ICP备14011364号</a>
<a href="http://www.beian.gov.cn/portal/registerSystemInfo?recordcode=44030402001866" target=_blank rel="noopener noreferer"><img style=display:inline-block src=/img/gaba.png>粤公网安备 44030402001866号</a>
</p>
<p id=webify class=powered-by style=display:flex><span style="margin:0 .2rem 0 0;display:block;height:20px;line-height:20px">Powered by</span> <a href=https://webify.cloudbase.net/ target=_blank rel=noopener>CloudBase Webify</a></p>
<script async defer>window.addEventListener('DOMContentLoaded',()=>{const{host:a}=window.location;(a.toLocaleLowerCase()==='docs.studyspiral.cn'||a.indexOf(".app.tcloudbase.com"))&&(document.getElementById('webify').style.display='flex')})</script>
</footer>
</main>
</div>
</div>
<script src=https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin=anonymous></script>
<script src=https://cdnjs.cloudflare.com/ajax/libs/jquery.imagesloaded/4.1.4/imagesloaded.pkgd.min.js integrity="sha256-lqvxZrPLtfffUl2G/e7szqSvPBILGbwmsGE1MKlOi0Q=" crossorigin=anonymous></script>
<script src=https://cdnjs.cloudflare.com/ajax/libs/jquery.isotope/3.0.6/isotope.pkgd.min.js integrity="sha256-CBrpuqrMhXwcLLUd5tvQ4euBHCdh7wGlDfNz8vbu/iI=" crossorigin=anonymous></script>
<script src=https://cdnjs.cloudflare.com/ajax/libs/fancybox/3.5.7/jquery.fancybox.min.js integrity="sha256-yt2kYMy0w8AbtF89WXb2P1rfjcP/HTHLT7097U8Y5b8=" crossorigin=anonymous></script>
<script src=https://cdnjs.cloudflare.com/ajax/libs/highlight.js/9.18.1/highlight.min.js integrity="sha256-eOgo0OtLL4cdq7RdwRUiGKLX9XsIJ7nGhWEKbohmVAQ=" crossorigin=anonymous></script>
<script src=https://cdnjs.cloudflare.com/ajax/libs/highlight.js/9.18.1/languages/go.min.js></script>
<script>const code_highlighting=!0</script>
<script>const isSiteThemeDark=!1</script>
<script>const search_config={indexURI:"/index.json",minLength:1,threshold:.3},i18n={no_results:"没有找到结果",placeholder:"搜索...",results:"搜索结果"},content_type={post:"文章",project:"项目",publication:"出版物",talk:"演讲"}</script>
<script src=https://cdnjs.cloudflare.com/ajax/libs/anchor-js/4.1.1/anchor.min.js integrity="sha256-pB/deHc9CGfFpJRjC43imB29Rse8tak+5eXqntO94ck=" crossorigin=anonymous></script>
<script>anchors.add()</script>
<script id=search-hit-fuse-template type=text/x-template>
      <div class="search-hit" id="summary-{{key}}">
      <div class="search-hit-content">
        <div class="search-hit-name">
          <a href="{{relpermalink}}">{{title}}</a>
          <div class="article-metadata search-hit-type">{{type}}</div>
          <p class="search-hit-description">{{snippet}}</p>
        </div>
      </div>
      </div>
    </script>
<script src=https://cdnjs.cloudflare.com/ajax/libs/fuse.js/3.2.1/fuse.min.js integrity="sha256-VzgmKYmhsGNNN4Ph1kMW+BjoYJM2jV5i4IlFoeZA9XI=" crossorigin=anonymous></script>
<script src=https://cdnjs.cloudflare.com/ajax/libs/mark.js/8.11.1/jquery.mark.min.js integrity="sha256-4HLtjeVgH0eIB3aZ9mLYF6E8oU5chNdjU6p6rrXpl9U=" crossorigin=anonymous></script>
<script src=/js/academic.min.6f1005d1a84220e2f466ff3d8e712f31.js></script>
<div id=modal class="modal fade" role=dialog>
<div class=modal-dialog>
<div class=modal-content>
<div class=modal-header>
<h5 class=modal-title>引用</h5>
<button type=button class=close data-dismiss=modal aria-label=Close>
<span aria-hidden=true>&#215;</span>
</button>
</div>
<div class=modal-body>
<pre><code class="tex hljs"></code></pre>
</div>
<div class=modal-footer>
<a class="btn btn-outline-primary my-1 js-copy-cite" href=# target=_blank>
<i class="fas fa-copy"></i> 复制
</a>
<a class="btn btn-outline-primary my-1 js-download-cite" href=# target=_blank>
<i class="fas fa-download"></i> 下载
</a>
<div id=modal-error></div>
</div>
</div>
</div>
</div>
</body>
</html>